{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "SUMMARY:\n",
    "Amazon pos/neg sentiment classification\n",
    "\n",
    "Accuracy: 0.94\n",
    "Time per Epoch: 9550 seconds = 220 rev/s\n",
    "Total time: 9550*10 = 1592 min = 26.5 hours\n",
    "Train size = 2,097,152\n",
    "Test size = 233,016\n",
    "\n",
    "DETAILS:\n",
    "Attempt to replicate crepe model using MXNET:\n",
    "https://github.com/zhangxiangxiao/Crepe\n",
    "\n",
    "This uses an efficient numpy array (dtype=bool)\n",
    "to hold all data in RAM. \n",
    "\n",
    "Run on one GPU (Tesla K80) with batch=128\n",
    "Peak RAM: 142GB, and training cut to: 2,097,152 (from 3.6M)\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import pickle\n",
    "import pandas as pd\n",
    "import mxnet as mx\n",
    "import wget\n",
    "import time\n",
    "import os.path\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "import logging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "AZ_ACC = \"amazonsentimenik\"\n",
    "AZ_CONTAINER = \"textclassificationdatasets\"\n",
    "\n",
    "ALPHABET = list(\"abcdefghijklmnopqrstuvwxyz0123456789-,;.!?:'\\\"/\\\\|_@#$%^&*~`+ =<>()[]{}\")\n",
    "FEATURE_LEN = 1014\n",
    "BATCH_SIZE = 128\n",
    "NUM_FILTERS = 256\n",
    "DATA_SHAPE = (BATCH_SIZE, 1, FEATURE_LEN, len(ALPHABET))\n",
    "\n",
    "ctx = mx.gpu(1)\n",
    "EPOCHS = 10\n",
    "SD = 0.05  # std for gaussian distribution\n",
    "NOUTPUT = 2  # good or bad\n",
    "INITY = mx.init.Normal(sigma=SD)\n",
    "LR = 0.01\n",
    "MOMENTUM = 0.9\n",
    "WDECAY = 0.00001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# logging\n",
    "logger = logging.getLogger()\n",
    "fhandler = logging.FileHandler(filename='crepe_amazon.log', mode='a')\n",
    "formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')\n",
    "fhandler.setFormatter(formatter)\n",
    "logger.addHandler(fhandler)\n",
    "logger.setLevel(logging.DEBUG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def download_file(url):\n",
    "    # Create file-name\n",
    "    local_filename = url.split('/')[-1]\n",
    "    if os.path.isfile(local_filename):\n",
    "        pass\n",
    "        # print(\"The file %s already exist in the current directory\\n\" % local_filename)\n",
    "    else:\n",
    "        # Download\n",
    "        print(\"downloading ...\\n\")\n",
    "        wget.download(url)\n",
    "        print('\\nsaved data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_data_frame(infile, shuffle = False):\n",
    "    print(\"processing data frame: %s\" % infile)\n",
    "    # Get data from windows blob\n",
    "    download_file('https://%s.blob.core.windows.net/%s/%s' % (AZ_ACC, AZ_CONTAINER, infile))\n",
    "    \n",
    "    # 3.6 mill is too much, use 2 mill (keep same ratio)\n",
    "    if \"test\" in infile:\n",
    "        maxrows = int(2097152/9)  # 16,384 batches\n",
    "    elif \"train\" in infile:\n",
    "        maxrows = int(2097152)\n",
    "\n",
    "    # load data into dataframe\n",
    "    df = pd.read_csv(infile,\n",
    "                     header=None,\n",
    "                     names=['sentiment', 'summary', 'text'],\n",
    "                     nrows=maxrows)\n",
    "    # Shuffle\n",
    "    if shuffle:\n",
    "        df = df.sample(frac=1).reset_index(drop=True)\n",
    "        \n",
    "    # concat summary, review; trim to 1014 char; reverse; lower\n",
    "    df['rev'] = df.apply(lambda x: \"%s %s\" % (x['summary'], x['text']), axis=1)\n",
    "    df.rev = df.rev.str[:FEATURE_LEN].str[::-1].str.lower()\n",
    "    \n",
    "    # store class as nparray\n",
    "    df.sentiment -= 1\n",
    "    y_split = np.asarray(df.sentiment, dtype='int')\n",
    "    # drop columns\n",
    "    df.drop(['text', 'summary', 'sentiment'], axis=1, inplace=True)\n",
    "\n",
    "    # Dictionary to create character vectors\n",
    "    character_hash = pd.DataFrame(np.identity(len(ALPHABET), dtype='bool'), columns=ALPHABET)\n",
    "    print(\"finished processing data frame: %s\" % infile)\n",
    "    print(\"data contains %d obs\" % df.shape[0])\n",
    "    batch_size = df.shape[0]\n",
    "    # Create encoding\n",
    "    X_split = np.zeros([batch_size, 1, FEATURE_LEN, len(ALPHABET)], dtype='bool')\n",
    "    # Main loop\n",
    "    for ti, tx in enumerate(df.rev):\n",
    "        if (ti+1) % (100*1000) == 0:\n",
    "            print(\"Processed: \", ti+1)\n",
    "        chars = list(tx)\n",
    "        for ci, ch in enumerate(chars):\n",
    "            if ch in ALPHABET:\n",
    "                X_split[ti % batch_size][0][ci] = np.array(character_hash[ch], dtype='bool')\n",
    "                \n",
    "    # Return as a DataBatch\n",
    "    #return DataBatch(data=[mx.nd.array(X_split)],\n",
    "    #                 label=[mx.nd.array(y_split[ti + 1 - batch_size:ti + 1])])\n",
    "    return X_split, y_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_crepe():\n",
    "    \"\"\"\n",
    "    Replicating: https://github.com/zhangxiangxiao/Crepe/blob/master/train/config.lua\n",
    "    \"\"\"\n",
    "    input_x = mx.sym.Variable('data')  # placeholder for input\n",
    "    input_y = mx.sym.Variable('softmax_label')  # placeholder for output\n",
    "    # 1. alphabet x 1014\n",
    "    conv1 = mx.symbol.Convolution(\n",
    "        data=input_x, kernel=(7, 69), num_filter=NUM_FILTERS)\n",
    "    relu1 = mx.symbol.Activation(\n",
    "        data=conv1, act_type=\"relu\")\n",
    "    pool1 = mx.symbol.Pooling(\n",
    "        data=relu1, pool_type=\"max\", kernel=(3, 1), stride=(3, 1))\n",
    "    # 2. 336 x 256\n",
    "    conv2 = mx.symbol.Convolution(\n",
    "        data=pool1, kernel=(7, 1), num_filter=NUM_FILTERS)\n",
    "    relu2 = mx.symbol.Activation(\n",
    "        data=conv2, act_type=\"relu\")\n",
    "    pool2 = mx.symbol.Pooling(\n",
    "        data=relu2, pool_type=\"max\", kernel=(3, 1), stride=(3, 1))\n",
    "    # 3. 110 x 256\n",
    "    conv3 = mx.symbol.Convolution(\n",
    "        data=pool2, kernel=(3, 1), num_filter=NUM_FILTERS)\n",
    "    relu3 = mx.symbol.Activation(\n",
    "        data=conv3, act_type=\"relu\")\n",
    "    # 4. 108 x 256\n",
    "    conv4 = mx.symbol.Convolution(\n",
    "        data=relu3, kernel=(3, 1), num_filter=NUM_FILTERS)\n",
    "    relu4 = mx.symbol.Activation(\n",
    "        data=conv4, act_type=\"relu\")\n",
    "    # 5. 106 x 256\n",
    "    conv5 = mx.symbol.Convolution(\n",
    "        data=relu4, kernel=(3, 1), num_filter=NUM_FILTERS)\n",
    "    relu5 = mx.symbol.Activation(\n",
    "        data=conv5, act_type=\"relu\")\n",
    "    # 6. 104 x 256\n",
    "    conv6 = mx.symbol.Convolution(\n",
    "        data=relu5, kernel=(3, 1), num_filter=NUM_FILTERS)\n",
    "    relu6 = mx.symbol.Activation(\n",
    "        data=conv6, act_type=\"relu\")\n",
    "    pool6 = mx.symbol.Pooling(\n",
    "        data=relu6, pool_type=\"max\", kernel=(3, 1), stride=(3, 1))\n",
    "    # 34 x 256\n",
    "    flatten = mx.symbol.Flatten(data=pool6)\n",
    "    # 7.  8704\n",
    "    fc1 = mx.symbol.FullyConnected(\n",
    "        data=flatten, num_hidden=1024)\n",
    "    act_fc1 = mx.symbol.Activation(\n",
    "        data=fc1, act_type=\"relu\")\n",
    "    drop1 = mx.sym.Dropout(act_fc1, p=0.5)\n",
    "    # 8. 1024\n",
    "    fc2 = mx.symbol.FullyConnected(\n",
    "        data=drop1, num_hidden=1024)\n",
    "    act_fc2 = mx.symbol.Activation(\n",
    "        data=fc2, act_type=\"relu\")\n",
    "    drop2 = mx.sym.Dropout(act_fc2, p=0.5)\n",
    "    # 9. 1024\n",
    "    fc3 = mx.symbol.FullyConnected(\n",
    "        data=drop2, num_hidden=NOUTPUT)\n",
    "    crepe = mx.symbol.SoftmaxOutput(\n",
    "        data=fc3, label=input_y, name=\"softmax\")\n",
    "    return crepe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\r\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\r\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\r\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\r\n",
       " -->\r\n",
       "<!-- Title: plot Pages: 1 -->\r\n",
       "<svg width=\"102pt\" height=\"2228pt\"\r\n",
       " viewBox=\"0.00 0.00 102.00 2228.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\r\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 2224)\">\r\n",
       "<title>plot</title>\r\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-2224 98,-2224 98,4 -4,4\"/>\r\n",
       "<!-- convolution0 -->\r\n",
       "<g id=\"node1\" class=\"node\"><title>convolution0</title>\r\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-58 -7.10543e-015,-58 -7.10543e-015,-0 94,-0 94,-58\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-32.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Convolution</text>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-17.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">7x69/1, 256</text>\r\n",
       "</g>\r\n",
       "<!-- activation0 -->\r\n",
       "<g id=\"node2\" class=\"node\"><title>activation0</title>\r\n",
       "<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"94,-152 -7.10543e-015,-152 -7.10543e-015,-94 94,-94 94,-152\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-126.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Activation</text>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-111.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">relu</text>\r\n",
       "</g>\r\n",
       "<!-- activation0&#45;&gt;convolution0 -->\r\n",
       "<g id=\"edge1\" class=\"edge\"><title>activation0&#45;&gt;convolution0</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-83.7443C47,-75.2043 47,-66.2977 47,-58.2479\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-93.8971 42.5001,-83.897 47,-88.8971 47.0001,-83.8971 47.0001,-83.8971 47.0001,-83.8971 47,-88.8971 51.5001,-83.8971 47,-93.8971 47,-93.8971\"/>\r\n",
       "</g>\r\n",
       "<!-- pooling0 -->\r\n",
       "<g id=\"node3\" class=\"node\"><title>pooling0</title>\r\n",
       "<polygon fill=\"#80b1d3\" stroke=\"black\" points=\"94,-246 -7.10543e-015,-246 -7.10543e-015,-188 94,-188 94,-246\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-220.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Pooling</text>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-205.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">max, 3x1/3</text>\r\n",
       "</g>\r\n",
       "<!-- pooling0&#45;&gt;activation0 -->\r\n",
       "<g id=\"edge2\" class=\"edge\"><title>pooling0&#45;&gt;activation0</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-177.744C47,-169.204 47,-160.298 47,-152.248\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-187.897 42.5001,-177.897 47,-182.897 47.0001,-177.897 47.0001,-177.897 47.0001,-177.897 47,-182.897 51.5001,-177.897 47,-187.897 47,-187.897\"/>\r\n",
       "</g>\r\n",
       "<!-- convolution1 -->\r\n",
       "<g id=\"node4\" class=\"node\"><title>convolution1</title>\r\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-340 -7.10543e-015,-340 -7.10543e-015,-282 94,-282 94,-340\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-314.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Convolution</text>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-299.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">7x1/1, 256</text>\r\n",
       "</g>\r\n",
       "<!-- convolution1&#45;&gt;pooling0 -->\r\n",
       "<g id=\"edge3\" class=\"edge\"><title>convolution1&#45;&gt;pooling0</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-271.744C47,-263.204 47,-254.298 47,-246.248\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-281.897 42.5001,-271.897 47,-276.897 47.0001,-271.897 47.0001,-271.897 47.0001,-271.897 47,-276.897 51.5001,-271.897 47,-281.897 47,-281.897\"/>\r\n",
       "</g>\r\n",
       "<!-- activation1 -->\r\n",
       "<g id=\"node5\" class=\"node\"><title>activation1</title>\r\n",
       "<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"94,-434 -7.10543e-015,-434 -7.10543e-015,-376 94,-376 94,-434\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-408.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Activation</text>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-393.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">relu</text>\r\n",
       "</g>\r\n",
       "<!-- activation1&#45;&gt;convolution1 -->\r\n",
       "<g id=\"edge4\" class=\"edge\"><title>activation1&#45;&gt;convolution1</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-365.744C47,-357.204 47,-348.298 47,-340.248\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-375.897 42.5001,-365.897 47,-370.897 47.0001,-365.897 47.0001,-365.897 47.0001,-365.897 47,-370.897 51.5001,-365.897 47,-375.897 47,-375.897\"/>\r\n",
       "</g>\r\n",
       "<!-- pooling1 -->\r\n",
       "<g id=\"node6\" class=\"node\"><title>pooling1</title>\r\n",
       "<polygon fill=\"#80b1d3\" stroke=\"black\" points=\"94,-528 -7.10543e-015,-528 -7.10543e-015,-470 94,-470 94,-528\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-502.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Pooling</text>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-487.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">max, 3x1/3</text>\r\n",
       "</g>\r\n",
       "<!-- pooling1&#45;&gt;activation1 -->\r\n",
       "<g id=\"edge5\" class=\"edge\"><title>pooling1&#45;&gt;activation1</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-459.744C47,-451.204 47,-442.298 47,-434.248\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-469.897 42.5001,-459.897 47,-464.897 47.0001,-459.897 47.0001,-459.897 47.0001,-459.897 47,-464.897 51.5001,-459.897 47,-469.897 47,-469.897\"/>\r\n",
       "</g>\r\n",
       "<!-- convolution2 -->\r\n",
       "<g id=\"node7\" class=\"node\"><title>convolution2</title>\r\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-622 -7.10543e-015,-622 -7.10543e-015,-564 94,-564 94,-622\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-596.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Convolution</text>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-581.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">3x1/1, 256</text>\r\n",
       "</g>\r\n",
       "<!-- convolution2&#45;&gt;pooling1 -->\r\n",
       "<g id=\"edge6\" class=\"edge\"><title>convolution2&#45;&gt;pooling1</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-553.744C47,-545.204 47,-536.298 47,-528.248\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-563.897 42.5001,-553.897 47,-558.897 47.0001,-553.897 47.0001,-553.897 47.0001,-553.897 47,-558.897 51.5001,-553.897 47,-563.897 47,-563.897\"/>\r\n",
       "</g>\r\n",
       "<!-- activation2 -->\r\n",
       "<g id=\"node8\" class=\"node\"><title>activation2</title>\r\n",
       "<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"94,-716 -7.10543e-015,-716 -7.10543e-015,-658 94,-658 94,-716\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-690.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Activation</text>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-675.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">relu</text>\r\n",
       "</g>\r\n",
       "<!-- activation2&#45;&gt;convolution2 -->\r\n",
       "<g id=\"edge7\" class=\"edge\"><title>activation2&#45;&gt;convolution2</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-647.744C47,-639.204 47,-630.298 47,-622.248\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-657.897 42.5001,-647.897 47,-652.897 47.0001,-647.897 47.0001,-647.897 47.0001,-647.897 47,-652.897 51.5001,-647.897 47,-657.897 47,-657.897\"/>\r\n",
       "</g>\r\n",
       "<!-- convolution3 -->\r\n",
       "<g id=\"node9\" class=\"node\"><title>convolution3</title>\r\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-810 -7.10543e-015,-810 -7.10543e-015,-752 94,-752 94,-810\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-784.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Convolution</text>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-769.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">3x1/1, 256</text>\r\n",
       "</g>\r\n",
       "<!-- convolution3&#45;&gt;activation2 -->\r\n",
       "<g id=\"edge8\" class=\"edge\"><title>convolution3&#45;&gt;activation2</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-741.744C47,-733.204 47,-724.298 47,-716.248\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-751.897 42.5001,-741.897 47,-746.897 47.0001,-741.897 47.0001,-741.897 47.0001,-741.897 47,-746.897 51.5001,-741.897 47,-751.897 47,-751.897\"/>\r\n",
       "</g>\r\n",
       "<!-- activation3 -->\r\n",
       "<g id=\"node10\" class=\"node\"><title>activation3</title>\r\n",
       "<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"94,-904 -7.10543e-015,-904 -7.10543e-015,-846 94,-846 94,-904\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-878.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Activation</text>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-863.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">relu</text>\r\n",
       "</g>\r\n",
       "<!-- activation3&#45;&gt;convolution3 -->\r\n",
       "<g id=\"edge9\" class=\"edge\"><title>activation3&#45;&gt;convolution3</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-835.744C47,-827.204 47,-818.298 47,-810.248\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-845.897 42.5001,-835.897 47,-840.897 47.0001,-835.897 47.0001,-835.897 47.0001,-835.897 47,-840.897 51.5001,-835.897 47,-845.897 47,-845.897\"/>\r\n",
       "</g>\r\n",
       "<!-- convolution4 -->\r\n",
       "<g id=\"node11\" class=\"node\"><title>convolution4</title>\r\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-998 -7.10543e-015,-998 -7.10543e-015,-940 94,-940 94,-998\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-972.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Convolution</text>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-957.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">3x1/1, 256</text>\r\n",
       "</g>\r\n",
       "<!-- convolution4&#45;&gt;activation3 -->\r\n",
       "<g id=\"edge10\" class=\"edge\"><title>convolution4&#45;&gt;activation3</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-929.744C47,-921.204 47,-912.298 47,-904.248\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-939.897 42.5001,-929.897 47,-934.897 47.0001,-929.897 47.0001,-929.897 47.0001,-929.897 47,-934.897 51.5001,-929.897 47,-939.897 47,-939.897\"/>\r\n",
       "</g>\r\n",
       "<!-- activation4 -->\r\n",
       "<g id=\"node12\" class=\"node\"><title>activation4</title>\r\n",
       "<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"94,-1092 -7.10543e-015,-1092 -7.10543e-015,-1034 94,-1034 94,-1092\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-1066.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Activation</text>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-1051.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">relu</text>\r\n",
       "</g>\r\n",
       "<!-- activation4&#45;&gt;convolution4 -->\r\n",
       "<g id=\"edge11\" class=\"edge\"><title>activation4&#45;&gt;convolution4</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-1023.74C47,-1015.2 47,-1006.3 47,-998.248\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-1033.9 42.5001,-1023.9 47,-1028.9 47.0001,-1023.9 47.0001,-1023.9 47.0001,-1023.9 47,-1028.9 51.5001,-1023.9 47,-1033.9 47,-1033.9\"/>\r\n",
       "</g>\r\n",
       "<!-- convolution5 -->\r\n",
       "<g id=\"node13\" class=\"node\"><title>convolution5</title>\r\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-1186 -7.10543e-015,-1186 -7.10543e-015,-1128 94,-1128 94,-1186\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-1160.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Convolution</text>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-1145.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">3x1/1, 256</text>\r\n",
       "</g>\r\n",
       "<!-- convolution5&#45;&gt;activation4 -->\r\n",
       "<g id=\"edge12\" class=\"edge\"><title>convolution5&#45;&gt;activation4</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-1117.74C47,-1109.2 47,-1100.3 47,-1092.25\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-1127.9 42.5001,-1117.9 47,-1122.9 47.0001,-1117.9 47.0001,-1117.9 47.0001,-1117.9 47,-1122.9 51.5001,-1117.9 47,-1127.9 47,-1127.9\"/>\r\n",
       "</g>\r\n",
       "<!-- activation5 -->\r\n",
       "<g id=\"node14\" class=\"node\"><title>activation5</title>\r\n",
       "<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"94,-1280 -7.10543e-015,-1280 -7.10543e-015,-1222 94,-1222 94,-1280\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-1254.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Activation</text>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-1239.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">relu</text>\r\n",
       "</g>\r\n",
       "<!-- activation5&#45;&gt;convolution5 -->\r\n",
       "<g id=\"edge13\" class=\"edge\"><title>activation5&#45;&gt;convolution5</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-1211.74C47,-1203.2 47,-1194.3 47,-1186.25\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-1221.9 42.5001,-1211.9 47,-1216.9 47.0001,-1211.9 47.0001,-1211.9 47.0001,-1211.9 47,-1216.9 51.5001,-1211.9 47,-1221.9 47,-1221.9\"/>\r\n",
       "</g>\r\n",
       "<!-- pooling2 -->\r\n",
       "<g id=\"node15\" class=\"node\"><title>pooling2</title>\r\n",
       "<polygon fill=\"#80b1d3\" stroke=\"black\" points=\"94,-1374 -7.10543e-015,-1374 -7.10543e-015,-1316 94,-1316 94,-1374\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-1348.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Pooling</text>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-1333.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">max, 3x1/3</text>\r\n",
       "</g>\r\n",
       "<!-- pooling2&#45;&gt;activation5 -->\r\n",
       "<g id=\"edge14\" class=\"edge\"><title>pooling2&#45;&gt;activation5</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-1305.74C47,-1297.2 47,-1288.3 47,-1280.25\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-1315.9 42.5001,-1305.9 47,-1310.9 47.0001,-1305.9 47.0001,-1305.9 47.0001,-1305.9 47,-1310.9 51.5001,-1305.9 47,-1315.9 47,-1315.9\"/>\r\n",
       "</g>\r\n",
       "<!-- flatten0 -->\r\n",
       "<g id=\"node16\" class=\"node\"><title>flatten0</title>\r\n",
       "<polygon fill=\"#fdb462\" stroke=\"black\" points=\"94,-1468 -7.10543e-015,-1468 -7.10543e-015,-1410 94,-1410 94,-1468\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-1435.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Flatten</text>\r\n",
       "</g>\r\n",
       "<!-- flatten0&#45;&gt;pooling2 -->\r\n",
       "<g id=\"edge15\" class=\"edge\"><title>flatten0&#45;&gt;pooling2</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-1399.74C47,-1391.2 47,-1382.3 47,-1374.25\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-1409.9 42.5001,-1399.9 47,-1404.9 47.0001,-1399.9 47.0001,-1399.9 47.0001,-1399.9 47,-1404.9 51.5001,-1399.9 47,-1409.9 47,-1409.9\"/>\r\n",
       "</g>\r\n",
       "<!-- fullyconnected0 -->\r\n",
       "<g id=\"node17\" class=\"node\"><title>fullyconnected0</title>\r\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-1562 -7.10543e-015,-1562 -7.10543e-015,-1504 94,-1504 94,-1562\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-1536.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">FullyConnected</text>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-1521.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">1024</text>\r\n",
       "</g>\r\n",
       "<!-- fullyconnected0&#45;&gt;flatten0 -->\r\n",
       "<g id=\"edge16\" class=\"edge\"><title>fullyconnected0&#45;&gt;flatten0</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-1493.74C47,-1485.2 47,-1476.3 47,-1468.25\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-1503.9 42.5001,-1493.9 47,-1498.9 47.0001,-1493.9 47.0001,-1493.9 47.0001,-1493.9 47,-1498.9 51.5001,-1493.9 47,-1503.9 47,-1503.9\"/>\r\n",
       "</g>\r\n",
       "<!-- activation6 -->\r\n",
       "<g id=\"node18\" class=\"node\"><title>activation6</title>\r\n",
       "<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"94,-1656 -7.10543e-015,-1656 -7.10543e-015,-1598 94,-1598 94,-1656\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-1630.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Activation</text>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-1615.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">relu</text>\r\n",
       "</g>\r\n",
       "<!-- activation6&#45;&gt;fullyconnected0 -->\r\n",
       "<g id=\"edge17\" class=\"edge\"><title>activation6&#45;&gt;fullyconnected0</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-1587.74C47,-1579.2 47,-1570.3 47,-1562.25\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-1597.9 42.5001,-1587.9 47,-1592.9 47.0001,-1587.9 47.0001,-1587.9 47.0001,-1587.9 47,-1592.9 51.5001,-1587.9 47,-1597.9 47,-1597.9\"/>\r\n",
       "</g>\r\n",
       "<!-- dropout0 -->\r\n",
       "<g id=\"node19\" class=\"node\"><title>dropout0</title>\r\n",
       "<polygon fill=\"#fccde5\" stroke=\"black\" points=\"94,-1750 -7.10543e-015,-1750 -7.10543e-015,-1692 94,-1692 94,-1750\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-1717.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Dropout</text>\r\n",
       "</g>\r\n",
       "<!-- dropout0&#45;&gt;activation6 -->\r\n",
       "<g id=\"edge18\" class=\"edge\"><title>dropout0&#45;&gt;activation6</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-1681.74C47,-1673.2 47,-1664.3 47,-1656.25\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-1691.9 42.5001,-1681.9 47,-1686.9 47.0001,-1681.9 47.0001,-1681.9 47.0001,-1681.9 47,-1686.9 51.5001,-1681.9 47,-1691.9 47,-1691.9\"/>\r\n",
       "</g>\r\n",
       "<!-- fullyconnected1 -->\r\n",
       "<g id=\"node20\" class=\"node\"><title>fullyconnected1</title>\r\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-1844 -7.10543e-015,-1844 -7.10543e-015,-1786 94,-1786 94,-1844\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-1818.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">FullyConnected</text>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-1803.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">1024</text>\r\n",
       "</g>\r\n",
       "<!-- fullyconnected1&#45;&gt;dropout0 -->\r\n",
       "<g id=\"edge19\" class=\"edge\"><title>fullyconnected1&#45;&gt;dropout0</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-1775.74C47,-1767.2 47,-1758.3 47,-1750.25\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-1785.9 42.5001,-1775.9 47,-1780.9 47.0001,-1775.9 47.0001,-1775.9 47.0001,-1775.9 47,-1780.9 51.5001,-1775.9 47,-1785.9 47,-1785.9\"/>\r\n",
       "</g>\r\n",
       "<!-- activation7 -->\r\n",
       "<g id=\"node21\" class=\"node\"><title>activation7</title>\r\n",
       "<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"94,-1938 -7.10543e-015,-1938 -7.10543e-015,-1880 94,-1880 94,-1938\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-1912.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Activation</text>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-1897.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">relu</text>\r\n",
       "</g>\r\n",
       "<!-- activation7&#45;&gt;fullyconnected1 -->\r\n",
       "<g id=\"edge20\" class=\"edge\"><title>activation7&#45;&gt;fullyconnected1</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-1869.74C47,-1861.2 47,-1852.3 47,-1844.25\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-1879.9 42.5001,-1869.9 47,-1874.9 47.0001,-1869.9 47.0001,-1869.9 47.0001,-1869.9 47,-1874.9 51.5001,-1869.9 47,-1879.9 47,-1879.9\"/>\r\n",
       "</g>\r\n",
       "<!-- dropout1 -->\r\n",
       "<g id=\"node22\" class=\"node\"><title>dropout1</title>\r\n",
       "<polygon fill=\"#fccde5\" stroke=\"black\" points=\"94,-2032 -7.10543e-015,-2032 -7.10543e-015,-1974 94,-1974 94,-2032\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-1999.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Dropout</text>\r\n",
       "</g>\r\n",
       "<!-- dropout1&#45;&gt;activation7 -->\r\n",
       "<g id=\"edge21\" class=\"edge\"><title>dropout1&#45;&gt;activation7</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-1963.74C47,-1955.2 47,-1946.3 47,-1938.25\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-1973.9 42.5001,-1963.9 47,-1968.9 47.0001,-1963.9 47.0001,-1963.9 47.0001,-1963.9 47,-1968.9 51.5001,-1963.9 47,-1973.9 47,-1973.9\"/>\r\n",
       "</g>\r\n",
       "<!-- fullyconnected2 -->\r\n",
       "<g id=\"node23\" class=\"node\"><title>fullyconnected2</title>\r\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-2126 -7.10543e-015,-2126 -7.10543e-015,-2068 94,-2068 94,-2126\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-2100.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">FullyConnected</text>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-2085.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">2</text>\r\n",
       "</g>\r\n",
       "<!-- fullyconnected2&#45;&gt;dropout1 -->\r\n",
       "<g id=\"edge22\" class=\"edge\"><title>fullyconnected2&#45;&gt;dropout1</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-2057.74C47,-2049.2 47,-2040.3 47,-2032.25\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-2067.9 42.5001,-2057.9 47,-2062.9 47.0001,-2057.9 47.0001,-2057.9 47.0001,-2057.9 47,-2062.9 51.5001,-2057.9 47,-2067.9 47,-2067.9\"/>\r\n",
       "</g>\r\n",
       "<!-- softmax -->\r\n",
       "<g id=\"node24\" class=\"node\"><title>softmax</title>\r\n",
       "<polygon fill=\"#fccde5\" stroke=\"black\" points=\"94,-2220 -7.10543e-015,-2220 -7.10543e-015,-2162 94,-2162 94,-2220\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-2187.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">SoftmaxOutput</text>\r\n",
       "</g>\r\n",
       "<!-- softmax&#45;&gt;fullyconnected2 -->\r\n",
       "<g id=\"edge23\" class=\"edge\"><title>softmax&#45;&gt;fullyconnected2</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-2151.74C47,-2143.2 47,-2134.3 47,-2126.25\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-2161.9 42.5001,-2151.9 47,-2156.9 47.0001,-2151.9 47.0001,-2151.9 47.0001,-2151.9 47,-2156.9 51.5001,-2151.9 47,-2161.9 47,-2161.9\"/>\r\n",
       "</g>\r\n",
       "</g>\r\n",
       "</svg>\r\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x3825cf8>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Visualise symbol (for crepe)\n",
    "crepe = create_crepe()\n",
    "\n",
    "a = mx.viz.plot_network(crepe)\n",
    "a.render('Crepe Model')\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_x, train_y = load_data_frame('amazon_review_polarity_train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(train_x.shape)\n",
    "print(train_y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_iter = mx.io.NDArrayIter(train_x, train_y, batch_size=BATCH_SIZE, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "del train_x\n",
    "del train_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = mx.model.FeedForward(\n",
    "    ctx = ctx,\n",
    "    symbol = create_crepe(), \n",
    "    num_epoch = EPOCHS,  # number of training rounds\n",
    "    learning_rate = LR,  # learning rate\n",
    "    momentum = MOMENTUM,   # momentum for sgd\n",
    "    wd = WDECAY,  # weight decay for reg\n",
    "    initializer = INITY  # init with sd of 0.05\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tic = time.time()\n",
    "model.fit(\n",
    "    X = train_iter,\n",
    "    eval_metric=['accuracy'],\n",
    "    batch_end_callback=mx.callback.Speedometer(BATCH_SIZE),\n",
    "    epoch_end_callback=mx.callback.do_checkpoint(\"crepe_checkp_\") \n",
    ")\n",
    "\n",
    "print(\"Finished training in %.0f seconds\" % (time.time() - tic))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "del train_iter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "# GPU broke after completing 7th epoch\n",
    "# Re-load checkpoint\n",
    "# If training breaks (happens on GPU), we can train further like so:\n",
    "\n",
    "# Load trained model:\n",
    "pretrained_model = mx.model.FeedForward.load(\"crepe_checkp_v2_\", 7)  \n",
    "\n",
    "model = mx.model.FeedForward(\n",
    "    ctx = ctx,\n",
    "    symbol=pretrained_model.symbol,\n",
    "    arg_params=pretrained_model.arg_params,\n",
    "    aux_params=pretrained_model.aux_params,\n",
    "    num_epoch=11, begin_epoch=7\n",
    ")\n",
    "\n",
    "# Train remaining epochs\n",
    "tic = time.time()\n",
    "model.fit(\n",
    "    X = train_iter,\n",
    "    eval_metric=['accuracy'],\n",
    "    batch_end_callback=mx.callback.Speedometer(100*BATCH_SIZE),\n",
    "    epoch_end_callback=mx.callback.do_checkpoint(\"crepe_checkp_v2_\") \n",
    ")\n",
    "\n",
    "print(\"Finished training in %.0f seconds\" % (time.time() - tic))\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Load test data\n",
    "test_x, test_y = load_data_frame('amazon_review_polarity_test.csv')\n",
    "test_iter = mx.io.NDArrayIter(test_x, test_y, batch_size=BATCH_SIZE, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Predict\n",
    "pred = np.argsort(model.predict(X = test_iter))[:,-1]\n",
    "\n",
    "# Save Results\n",
    "np.savetxt('crepe_predict_sentiment_amazon.csv', np.c_[pred, test_y], delimiter=',', fmt='%d')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.94166495004634876"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Accuracy\n",
    "acc = sum(pred==test_y.astype('int'))/float(len(test_y))\n",
    "logger.info(acc)\n",
    "acc  # 0.94"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Log Extract\n",
    "\n",
    "\n",
    "```\n",
    "2016-08-24 21:11:28,407 - root - INFO - Start training with [gpu(0)]\n",
    "2016-08-24 21:12:40,015 - root - INFO - Epoch[0] Batch [50]\t Speed: 114.7797 samples/sec\tTrain-accuracy=0.502031\n",
    "2016-08-24 21:13:15,326 - root - INFO - Epoch[0] Batch [100]\tSpeed: 181.3185 samples/sec\tTrain-accuracy=0.514531\n",
    "2016-08-24 21:13:46,977 - root - INFO - Epoch[0] Batch [150]\tSpeed: 202.2053 samples/sec\tTrain-accuracy=0.515938\n",
    "2016-08-24 21:14:17,447 - root - INFO - Epoch[0] Batch [200]\tSpeed: 210.0496 samples/sec\tTrain-accuracy=0.522344\n",
    "2016-08-24 21:14:48,170 - root - INFO - Epoch[0] Batch [250]\tSpeed: 208.3130 samples/sec\tTrain-accuracy=0.540156\n",
    "...\n",
    "2016-08-26 20:53:13,394 - root - INFO - Epoch[10] Batch [16200]\tSpeed: 216.1578 samples/sec\tTrain-accuracy=0.967656\n",
    "2016-08-26 20:53:43,127 - root - INFO - Epoch[10] Batch [16250]\tSpeed: 215.2490 samples/sec\tTrain-accuracy=0.970313\n",
    "2016-08-26 20:54:11,734 - root - INFO - Epoch[10] Batch [16300]\tSpeed: 223.7136 samples/sec\tTrain-accuracy=0.967812\n",
    "2016-08-26 20:54:40,546 - root - INFO - Epoch[10] Batch [16350]\tSpeed: 222.1374 samples/sec\tTrain-accuracy=0.971250\n",
    "2016-08-26 20:55:00,513 - root - INFO - Epoch[10] Resetting Data Iterator\n",
    "2016-08-26 20:55:00,513 - root - INFO - Epoch[10] Time cost=9474.580\n",
    "\n",
    "2016-08-26 21:16:02,765 - root - INFO - 0.941664950046\n",
    "```"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
